package Crawler;

import com.google.common.base.Joiner;
import com.mongodb.BasicDBObject;
import com.mongodb.DB;
import com.mongodb.DBCollection;
import com.mongodb.MongoClient;
import org.apache.commons.io.FileUtils;
import org.jsoup.Connection;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import redis.clients.jedis.Jedis;

import java.io.*;
import java.net.UnknownHostException;
import java.util.ArrayList;
import java.util.Enumeration;
import java.util.List;
import java.util.StringTokenizer;

import static jdk.nashorn.internal.runtime.regexp.joni.Syntax.Java;

/**
 * 在原来文件的基础上，增加redis以及IP代理
 */

public class NewSpider {

    private static Jedis jedis = new Jedis("222.27.227.117", 6379);
    private static MongoClient mongo = new MongoClient("222.27.227.24", 27017);

    private static IPPoolClass ippoolThread;


    // 静态代码块：用于进行初始化的操作
    static {

        jedis.select(0);

        // 初始化IP池自动更新的线程
        System.out.println("[DEBUG] ： 代码初始化");
        ippoolThread = new IPPoolClass();
        ippoolThread.start();
        System.out.println("[DEBUG] : IP池更新线程已成功初始化");

        // 等待第一次IP池装载
        try {
            Thread.sleep(10000);
        } catch (Exception ee) {
            System.out.println(ee.getStackTrace());
        }
    }

    // 连接MONGODB并存入数据
    public static void writeDataIntoMongdb(BasicDBObject document) {

        // DBObject接口和BasicDBObject对象：表示一个具体的记录，BasicDBObject实现了DBObject，是key-value的数据结构，用起来和HashMap是基本一致的。
        DB db = mongo.getDB("WEIBO");
        DBCollection table = db.getCollection("follower");
        table.save(document);
        System.out.println("数据已入库");
    }


    // 连接Redis,从中获取uid以及Connum,返回的是取出的一条数据
    public static String getUidAndConnumIntoRedis() {

        // 从Redis中获取数据并出队，uid以及connum要分开
        // 不能用队列一次取出，只能取出一个然后销毁
        String element = jedis.lpop("follower:uid_connum");
        System.out.println("当前使用了: " + element);
        System.out.println("现出队--------------------" + element);
        return element;
    }


//    // 加入IP代理
//    public static void addIPAgency() {
//
//        // 设置最大请求次数
//        System.setProperty("http.maxRedirects", "200");
//
//        // 设置系统参数，如果代理IP端口正确也可以不加
//        System.getProperties().setProperty("proxySet", "true");
//
//        // 从redis中取出ip和端口号,然后分开
//        List<String> list = jedis.lrange("weibo:ippool", 0, -1);
//        for (int i = 0; i < list.size(); i++) {
//            String line = list.get(i).toString();
//            String[] parts = line.split(":");
//            String ip = parts[0];
//            String port = parts[1];
//            System.getProperties().setProperty("http.proxyHost", ip);
//            System.getProperties().setProperty("http.proxyPort", port);
//        }
//        System.out.println("IP代理已就位......");
//    }

    // 设置当前IP池随机的一个IP代理
    public static void setIP() {
        // 设置最大请求重试次数
        System.setProperty("http.maxRedirects", "200");
        // 设置系统参数
        System.getProperties().setProperty("proxySet", "true");
        // 获取到当前的IP池
        List<String> curIppool = ippoolThread.getIPPOOL();
        // 获取当前IP池的长度
        Integer ippoolSize = curIppool.size();
        // 从当前的IP池中随机抽取一个
        double randomIndex = ippoolSize * Math.random();
        String line = curIppool.get((int)randomIndex);
        // 将随机抽取到的IP进行处理和设置用于本次请求
        String[] parts = line.split(":");
        String ip = parts[0];
        String port = parts[1];
        System.getProperties().setProperty("http.proxyHost", ip);
        System.getProperties().setProperty("http.proxyPort", port);
    }

    // 主函数
    public static void main(String[] args) {

        // UID计数初始化
        int count = 1;

        // 设置循环标记
        boolean flag = true;

        while (flag) {
            System.out.println("爬取第" + count + "个UID");
            // 获取所处理的uid——connum
            String uid_connum = getUidAndConnumIntoRedis();
            if (null == uid_connum) {
                flag = false;
            }
            // 将UID以及CONNUM分开
            String[] parts = uid_connum.split(",");
            String uid = parts[0];
            // connums   Integer.parseInt输出一个十进制的数字，radix表述逗号前面的是什么进制的数字，无论参数是什么进制输出都是十进制
            int following = Integer.parseInt(parts[1], 10);
            crawler_and_toMongo(uid, following);
        }

    }


    // 获取文件的一行
    public static ArrayList getUidList(String fileName) {
        // 将文件的一行变为数组
        ArrayList uidList = new ArrayList();
        try {
            for (String line : FileUtils.readLines(new File(fileName))) {
                uidList.add(line);
            }
        } catch (IOException e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
        }
        return uidList;
    }

    // 随机获取时间段
    public static void timeDelay(float min, float max) {
        int random = (int) (max * Math.random() + min);
        try {
            Thread.sleep(random * 1000);
        } catch (InterruptedException e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
        }
    }

    //数据爬取以及数据的读入
    public static void crawler_and_toMongo(String uid, int following) {

        // 微博关注页数中最大展示关注数一页32个
        float total_number = (float) (following * 1.0 / 32 + 3);
        ArrayList following_list = new ArrayList();

        for (int i = 1; i < total_number; i++) {
            System.out.println("The current uid is " + uid + " and the page number is " + i);
            String following_url = "http://tw.weibo.com/" + uid + "/follow/p/" + i;
            // Jsoup 是一个 Java 的开源HTML解析器，可直接解析某个URL地址、HTML文本内容
            // 获取请求连接
            Connection con = Jsoup.connect("http://tw.weibo.com/api/user/follow");
            // data请求参数
            con.data("uid", uid);
            con.data("page", String.valueOf(i));
            con.data("currentuid", "1991953413");
            con.data("page_size", "32");
            // 请求头设置
            con.header("Cookie",
                    "SINAGLOBAL=9711952802808.682.1499668084041; __gads=ID=130480cbf75a5ad6:T=1500258878:S=ALNI_MZRcFnJveJoxtD4TBbJNbD__KcD-A; UM_distinctid=15d4e658b271f5-0573ee5fbac68c-1b1d7751-1fa400-15d4e658b28944; wvr=6; UOR=,,www.shejidaren.com; _ga=GA1.2.84290483.1500258858; _gid=GA1.2.952496907.1501988163; crtg_rta=; SSOLoginState=1502013392; SCF=Agl4g4wcoIanRFGqnVTCrYVRKcGcCSGCXik1olhn6h9oPJpHfvjbBZyqPuHU-ibuQXyHNclynoHBANDkf_WRtWc.; SUB=_2A250gpeCDeRhGedH4lMY9S3Nzz-IHXVX-Y5KrDV8PUNbmtBeLUWjkW9XZkSwKQAYTJkZTvK42M-ekE_Hnw..; SUBP=0033WrSXqPxfM725Ws9jqgMF55529P9D9WWEbrh.i5bvbnSvvbQHi2M35JpX5KMhUgL.Fo241K24SKepShe2dJLoIpjLxKML1KBLBo5LxK-LB--LB-2LxK-L1h-L1h2t; SUHB=0LcjYQm8zbqhfQ; ALF=1533549388; _s_tentry=-; Apache=9574205823039.291.1502013327944; ULV=1502013327950:6:1:2:9574205823039.291.1502013327944:1501421673576");
            // con.header("Referer","http://tw.weibo.com/libingbing/follow/p/12");
            con.header("Referer", following_url);
            Document doc;
            try {
                // 加入IP代理
//                addIPAgency();
                setIP();
                // 解析请求结果post
                doc = con.post();
                System.out.println(doc);
                for (Element e : doc.select("[class =picBox]")) {
                    String following_uid = e.getElementsByTag("div").attr("uid");
                    // System.out.println(following_uid);
                    following_list.add(following_uid);
                }
                // 请求时间在0-2中随机抽取一个数字5-15  30-35
                timeDelay(0, 2);
            } catch (IOException e1) {
                e1.printStackTrace();
                timeDelay(30, 35);
                // continue是跳过当次循环中剩下的语句，执行下一次循环。
                continue;
            }
        }
        String uids = Joiner.on(",").join(following_list);
        // System.out.println("The uids is "+uids);
        BasicDBObject document = new BasicDBObject();
        document.put("_id", uid);
        document.put("following", uids);

        writeDataIntoMongdb(document);
    }


    // 内部类： 固定时间间隔更新IP池
    static class IPPoolClass extends Thread {

        // IP池
        private List<String> IPPOOL;

        public List<String> getIPPOOL() {
            return this.IPPOOL;
        }

        // IP池的更新间隔
        final Integer REFRESH_TIME_INTERVAL = 5;

        Jedis ippoolJedis = new Jedis("222.27.227.117", 6379);

        // 实时更新IP池
        public void run() {
            while (true) {
                // 更新IP池
                IPPOOL = ippoolJedis.lrange("weibo:ippool", 0, -1);
                System.out.println("[INFO] : IP池更新成功！当前IP池大小为 [ " + IPPOOL.size() + " ] .");
                // 等待
                try {
                    Thread.sleep(REFRESH_TIME_INTERVAL * 1000);
                } catch (Exception ee) {
                    System.out.println(ee.getStackTrace());
                }
            }

        }

    }


}
